% Mabel Zhang
% 28 Nov 2011

%function [data_pruned] = prune_cv (train)
% prune features that are not useful
% Parameters:
%   train: data to prune. Could be train or test data


%% Combine upper and lower case words
% capitalized words - combine with lower case, in vocab and in train (or X)?

%{

% Loop through vocab, find words that are the same in lower case, merge them
for i = 1 : numel (vocab)
  vocab{i} = lower(vocab{i});
end

% Sort vocab, save the original index
[vocab_sorted, idx_sorted] = sort(vocab);

equi_idx = [];

% Delete duplicate words
% Should only take linear time. If don't sort, will need to compare with
%   n^2 time.
for i = 1 : numel(vocab_sorted)-1
  
  % If two words are same, remove the second one
  if strcmp(vocab_sorted{i}, vocab_sorted{i+1}) == 1
    
    % Find orig vocab index for these two words
    keepword_vocab_idx = idx_sorted(i);
    dupword_vocab_idx = idx_sorted(i+1);
    
    % Save the original index to remove
    equi_idx (numel (equi_idx) + 1) = dupword_vocab_idx;
    
    % Remove the dup word in training data
    for j = 1 : numel(train)
      
      % If the keep word is found in this sample
      keep_idxInEx = find (train(j).word_idx == keepword_vocab_idx);
      % If the dup word is found in this sample
      dup_idxInEx = find (train(j).word_idx == dupword_vocab_idx);
      
      % If both keep word and dup word are found in this sample, sum their
      %   counts, discard dup word
      if (numel (keep_idxInEx) > 0) && (numel (dup_idxInEx) > 0)
       
        % Add dup word's count to keepword
        train(j).word_count(keep_idxInEx) = train(j).word_count(keep_idxInEx) ...
          + train(j).word_count(dup_idxInEx);
        
        % Delete the entries for the dupword
        train(j).word_idx(dup_idxInEx) = [];
        train(j).word_count(dup_idxInEx) = [];
        
      % If training data has ONLY the dup word, replace index with the
      %   first
      elseif numel (dup_idxInEx) > 0
        % Set the word index in dup word to keepword's word index
        train(j).word_idx(dup_idxInEx) = keepword_vocab_idx;
      end
    end
    
  end
end

% Remove dup words from ORIGINAL vocab (`.` indices are in original vocab's
%   index, not in the dup one)
vocab(equi_idx) = [];

clear equi_idx;

%}


%% Prune stopwords and numbers
% numerical strings - 1, 2, 3, 4, 5, 10, remove all

% articles(a, the), personal pronouns (I, my, mine), conjunctions (and, or),
%   contractions(s, t), prepositions
stopwords = {'a', 'an', 'the', ...
  'i', 'me', 'my', 'mine', ...
  'you', 'your', 'yours', 'yourself', 'yourselves', ...
  'he', 'him', 'his', 'she', 'her', 'hers', ...
  'we', 'us', 'our', 'ourselves', ...
  'they', 'them', 'their', 'theirs', 'themselves', ...
  'and', 'or', ...
  's', 't', 'm', ...
  'to', 'by', 'in', 'on', 'from', 'for', 'as', 'at', 'of', 'so'...
  'some', 'then', ...
  'be', 'is', 'are', ...
  'do', 'did', 'does', ...
  'has', 'had', 'have', ...
  'may', ...
  'when', 'who', 'where', 'which', 'what'};

remove_idx = [];

%isstrprop(vocab, 'digit');
% If this word is entirely a number
%if numel(isnum{i}) == sum(isnum{i})

% Find all vocabs that are numeric, record the indices
for i = 1 : numel (vocab)
  
  % If word is a number
  %if ~isempty (str2num (vocab{i}))
  if ~isnan (str2double (vocab{i}))
    disp(vocab{i});
    
    % Record this index to remove later
    remove_idx (numel(remove_idx) + 1) = i;
    
  % If word is a stopword
  elseif numel (strmatch (vocab{i}, stopwords, 'exact')) > 0
    disp(vocab{i});
    
    % Record this index to remove later
    remove_idx (numel(remove_idx) + 1) = i;
    
  end
end

% Remove stopword and numeric words from train data
for j = 1 : numel (train)
%for j = 1 : numel (test)
  for i = 1 : numel (remove_idx)
    
    % If this example has this stopword, remove it
    idxInEx = find (train(j).word_idx == remove_idx(i));
    %idxInEx = find (test(j).word_idx == remove_idx(i));
  
    if numel (idxInEx) > 0
      train(j).word_idx (idxInEx) = [];
      train(j).word_count (idxInEx) = [];
      %test(j).word_idx (idxInEx) = [];
      %test(j).word_count (idxInEx) = [];
    end
    
  end
end

% Remove numerical strings from voab
vocab(remove_idx) = [];

clear remove_idx;



%end

